import domain.Page;
import get.HtmlGet;
import get.impl.DefaultHtmlGetter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import output.OutPut;
import output.impl.ConsoleOutPut;
import parser.Parse;
import parser.impl.JDParser;
import urlBase.URLBase;
import urlBase.impl.RedisBase;

import java.io.IOException;

public class Crawler {

    public HtmlGet getter;
    public Parse parser;
    public OutPut output;

    public Crawler(){
        this.getter = new DefaultHtmlGetter();
        this.parser = new JDParser();
        this.output = new ConsoleOutPut();
    }
    public Crawler(HtmlGet getter,Parse parser,OutPut output)
    {
        this.getter=getter;
        this.parser=parser;
        this.output=output;
    }

    public Page get(String url)
    {
        return getter.get(url);
    }

    public void parse(Page page)
    {
        parser.parse(page);
    }

    public void output(Page page)
    {
        output.output(page);
    }

    public void start() throws IOException, InterruptedException {
        URLBase urlbase = RedisBase.getURLBase();
        urlbase.addIndex("https://www.jd.com/allSort.aspx");
        while (true) {
            String index = urlbase.getIndex();
            urlbase.discard(index);
            Document doc = Jsoup.connect(index).get();
            Thread.sleep(100);
            doc.select("a[href]")
                    .forEach(element ->
                    {
                        String href = element.attr("href");
                        if (!href.startsWith("https:")) {
                            href = "https:" + href;
                        }
                        if (href.matches("https://list.jd.com/.*")) {
                            try {
                                Thread.sleep(100);
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }
                            if (!urlbase.exist(href)) {
                                urlbase.addIndex(href);
                            }
                        } else if (href.matches("https://item.jd.com/.*.html")) {
                            try {
                                Thread.sleep(100);
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }
                            if (!urlbase.exist(href)) {
                                urlbase.addItem(href);
                            }
                        }
                    });
            Thread.sleep(1000);
            for(int i =0; i<urlbase.size();i++)
            {
                String itemURL = urlbase.getItem();
                if(!urlbase.exist(itemURL))
                {
                    urlbase.discard(itemURL);
                    Page page = get(itemURL);
                    parse(page);
                    System.out.println(page.getTitle());
//                    output(page);
                }
                urlbase.discard(itemURL);
            }
        }
    }

    public static void main(String[] args) {
        Crawler crawler = new Crawler();
        try {
            crawler.start();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
